{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 导包\n",
    "import collections\n",
    "import math\n",
    "import os\n",
    "import random\n",
    "import zipfile\n",
    "import numpy as np\n",
    "import urllib\n",
    "import tensorflow as tf\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_words(data_df, feature = 'words'):\n",
    "    data_df = data_df[feature].values\n",
    "    dictionary = []\n",
    "    for sent in data_df:\n",
    "        sent = sent[1:-1]\n",
    "        sent = sent.split(';')\n",
    "        for word in sent:\n",
    "            if word != '':\n",
    "                dictionary.append(word)\n",
    "    return dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3169331 ['好', '大', '的', '一个', '游乐', '公园', '已经', '去', '了', '2', '次', '但', '感觉', '还', '没有', '玩', '够', '似的', '会', '有', '第', '三', '第', '四', '次', '的', '新中国', '成立', '也', '是', '在', '这', '举行', '对', '我们', '中国人', '来说', '有些', '重要', '及', '深刻', '的', '意义', '庐山', '瀑布', '非常', '有名', '也', '有', '非常', '多', '个', '瀑布', '只是', '最', '好看', '的', '非', '三叠泉', '莫', '属', '推荐', '一', '去', '个人', '觉得', '颐和园', '是', '北京', '最', '值', '的', '一起', '的', '地方', '不过', '相比', '下', '门票', '也', '是', '最贵', '的', '比起', '故宫', '的', '雄伟', '与', '气势磅礴', '颐和园', '的', '宁静', '与', '波光粼粼', '更加', '美', '吧', '~', '迪斯尼', '一日游']\n"
     ]
    }
   ],
   "source": [
    "# hancks 分词\n",
    "model_path = '../input/'\n",
    "\n",
    "train_word = pd.read_csv(model_path + 'train_word.csv')\n",
    "test_word = pd.read_csv(model_path + 'predict_word.csv')\n",
    "data_word = pd.concat([train_word, test_word])\n",
    "    \n",
    "words = get_words(data_word, feature = 'words')\n",
    "print(len(words), words[0:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocabulary_size = 50000\n",
    "def build_dataset(words):\n",
    "    count = [['UNK', -1]]\n",
    "    count.extend(collections.Counter(words).most_common(vocabulary_size - 1))\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        if word in dictionary:\n",
    "            index = dictionary[word]\n",
    "        else:\n",
    "            index = 0\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reverse_dictionary\n",
    "\n",
    "data, count, dictionary, reverse_dictionary = build_dataset(words) # 用编号存储 可以节省内存"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most common words (+UNK) [['UNK', 46942], ('的', 206104), ('是', 54711), ('了', 54278), ('去', 41036)]\n",
      "Sample data [25, 66, 1, 22, 1178, 143, 161, 4, 3, 171] ['好', '大', '的', '一个', '游乐', '公园', '已经', '去', '了', '2']\n"
     ]
    }
   ],
   "source": [
    "del words\n",
    "print('Most common words (+UNK)', count[:5])\n",
    "print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_index = 0\n",
    "\n",
    "def generate_batch(batch_size, num_skips, skip_window):\n",
    "    global data_index\n",
    "    assert batch_size % num_skips == 0\n",
    "    assert num_skips <= 2 * skip_window\n",
    "    batch = np.ndarray(shape = (batch_size), dtype = np.int32) # 行向量\n",
    "    labels = np.ndarray(shape = (batch_size, 1), dtype  = np.int32) # 列向量\n",
    "    span = 2 * skip_window + 1\n",
    "    buffer = collections.deque(maxlen = span)\n",
    "    for _ in range(span):\n",
    "        buffer.append(data[data_index])\n",
    "        data_index = (data_index + 1) % len(data)\n",
    "    for i in range(batch_size // num_skips):\n",
    "        target = skip_window\n",
    "        targets_to_avoid = [skip_window]\n",
    "        for j in range(num_skips):\n",
    "            while target in targets_to_avoid:\n",
    "                target = random.randint(0, span - 1)\n",
    "            targets_to_avoid.append(target)\n",
    "            batch[i * num_skips + j] = buffer[skip_window]\n",
    "            labels[i * num_skips + j, 0] = buffer[target]\n",
    "        buffer.append(data[data_index])\n",
    "        data_index = (data_index + 1) % len(data)\n",
    "    return batch, labels\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66 大 -> 25 好\n",
      "66 大 -> 1 的\n",
      "1 的 -> 22 一个\n",
      "1 的 -> 66 大\n",
      "22 一个 -> 1178 游乐\n",
      "22 一个 -> 1 的\n",
      "1178 游乐 -> 22 一个\n",
      "1178 游乐 -> 143 公园\n"
     ]
    }
   ],
   "source": [
    "batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)\n",
    "for i in range(8):\n",
    "    print(batch[i], reverse_dictionary[batch[i]], '->', labels[i, 0], reverse_dictionary[labels[i, 0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "embedding_size = 128\n",
    "skip_window = 1\n",
    "num_skips = 2\n",
    "\n",
    "valid_size = 16\n",
    "valid_window = 100\n",
    "valid＿examples = np.random.choice(valid_window, valid_size, replace=False) # [0, valid_window) 的随机数 不重复 len = valid_size\n",
    "num_sampled = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    train_inputs = tf.placeholder(tf.int32, shape = [batch_size])\n",
    "    train_labels = tf.placeholder(tf.int32, shape = [batch_size, 1])\n",
    "    valid_dataset = tf.constant(valid_examples, dtype = tf.int32)\n",
    "    embeddings = tf.Variable(\n",
    "                tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))\n",
    "    embed = tf.nn.embedding_lookup(embeddings, train_inputs)\n",
    "    nce_weights = tf.Variable(\n",
    "                tf.truncated_normal([vocabulary_size, embedding_size], \n",
    "                                   stddev = 1.0 / math.sqrt(embedding_size)))\n",
    "    nce_biases = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "    loss = tf.reduce_mean(tf.nn.nce_loss(weights=nce_weights,\n",
    "                                         biases=nce_biases,\n",
    "                                         labels=train_labels,\n",
    "                                         inputs=embed,\n",
    "                                         num_sampled=num_sampled,\n",
    "                                         num_classes=vocabulary_size))\n",
    "    optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)\n",
    "    norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))\n",
    "    normalized_embeddings = embeddings / norm\n",
    "    valid_embeddings =tf.nn.embedding_lookup(\n",
    "        normalized_embeddings, valid_dataset)\n",
    "    similarity = tf.matmul(\n",
    "        valid_embeddings, normalized_embeddings, transpose_b=True)\n",
    "    init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Average loss at step 0 :  294.903564453125\n",
      "Nearest to 等:  西对,  SPA,  吹发,  灯会,  白马涧,  對於,  大元帅,  花雨, \n",
      "Nearest to 去:  水殿,  1.4,  朗朗上口,  坐船来,  上下,  慎,  氧气瓶,  拆卸, \n",
      "Nearest to 这个:  外人,  2.3,  昆明,  雪花,  宝冠,  黄线,  短途,  穿衣服, \n",
      "Nearest to 西湖:  自然保护区,  里能,  灵兽,  够劲,  母亲节,  宋真宗,  南广场,  整个, \n",
      "Nearest to 最:  藉,  翘角,  武当,  没赶上,  摸着,  溜溜,  中小,  适应, \n",
      "Nearest to 有:  鎏金,  揣,  沟沟,  形制,  基本相同,  砥砺,  那一次,  西部, \n",
      "Nearest to 但:  捐,  伤痕,  铁栏杆,  腹心,  吴隐,  古塔,  设立,  有客栈, \n",
      "Nearest to 这里:  慵懒,  有机,  水乡好点,  不卖,  夸张,  岱山,  胎,  雄壮, \n",
      "Nearest to 两:  乡试,  >n--,  回头,  少少,  基金,  堆成,  太阳石,  桃, \n",
      "Nearest to 不错:  冰上,  上官,  三十三,  太酷,  负,  仅,  稍,  航空公司网站, \n",
      "Nearest to 特别:  土砖,  山谷,  以备,  节日期间,  拒人千里,  小王子,  岭南,  比赛, \n",
      "Nearest to 看看:  耳熟能详,  联想起,  扬州,  爱情,  石景山,  那多,  贪吃,  山间, \n",
      "Nearest to 了:  坝顶,  哪知,  井冈山革命博物馆,  中国,  西魏,  倒插,  彷佛,  滑润, \n",
      "Nearest to 能:  平台,  地理书,  参天大树,  吉普车,  逐,  附身,  盐池,  6666, \n",
      "Nearest to 因为:  总投资,  族规,  秦勇,  嫂,  飞来为,  降临,  快照,  调味, \n",
      "Nearest to 不是:  山和云,  抽走,  大佛像,  玉渊潭公园,  淮北,  翻造,  梨木,  大华山镇, \n",
      "Average loss at step 2000 :  117.89948826503753\n",
      "Average loss at step 4000 :  55.03231735539436\n",
      "Average loss at step 6000 :  34.90973995518684\n",
      "Average loss at step 8000 :  24.736747129678726\n",
      "Average loss at step 10000 :  18.3757007843256\n",
      "Nearest to 等:  z,  很多,  美的,  体现,  琼台,  人造,  佛光,  灯会, \n",
      "Nearest to 去:  来,  二级,  一大早,  受不了,  份,  在,  定时,  水乡, \n",
      "Nearest to 这个:  黄线,  外人,  观看,  其他,  雪花,  省,  昆明,  穿衣服, \n",
      "Nearest to 西湖:  自然保护区,  拱券,  数,  整个,  大修,  灯火阑珊,  留,  灵兽, \n",
      "Nearest to 最:  武当,  的,  没赶上,  全部,  佛祖,  艺人,  纪晓岚,  色, \n",
      "Nearest to 有:  的,  是,  鎏金,  还有,  黑马河乡,  了,  珍稀动物,  在, \n",
      "Nearest to 但:  设立,  老街,  喜欢,  死人,  展园,  的,  体,  铁栏杆, \n",
      "Nearest to 这里:  有机,  不收,  巅,  慵懒,  夸张,  长沙,  辉,  美得, \n",
      "Nearest to 两:  中秋,  附件,  瑰宝,  自古以来,  幽,  银川,  祭堂,  阁, \n",
      "Nearest to 不错:  古国,  鬼,  不须,  高校,  稍,  那里,  挺好,  来, \n",
      "Nearest to 特别:  比赛,  很,  山谷,  众,  超级,  头,  多种,  偏, \n",
      "Nearest to 看看:  扬州,  耳熟能详,  爱情,  石景山,  拓展,  神龙,  升旗仪式,  而已, \n",
      "Nearest to 了:  的,  有,  古国,  摇橹船,  就,  可以,  三河,  是, \n",
      "Nearest to 能:  平台,  吉普车,  参天大树,  算得,  存在,  自作主张,  可以,  我们, \n",
      "Nearest to 因为:  总投资,  快照,  作为,  有奖,  中药材,  最高峰,  120,  大足石刻, \n",
      "Nearest to 不是:  宗教,  餐,  比,  玉渊潭公园,  仙女山,  很长,  鸣沙山,  宏村, \n",
      "Average loss at step 12000 :  14.597306035757065\n",
      "Average loss at step 14000 :  12.075511926412583\n",
      "Average loss at step 16000 :  10.328506790161132\n",
      "Average loss at step 18000 :  9.381531036376954\n",
      "Average loss at step 20000 :  8.380836192011833\n",
      "Nearest to 等:  很多,  z,  和,  琼台,  人造,  体现,  美的,  开船, \n",
      "Nearest to 去:  来,  擦擦,  引桥,  到,  二级,  转车,  越王,  一大早, \n",
      "Nearest to 这个:  黄线,  其他,  外人,  雪花,  穿衣服,  观看,  我们,  十分, \n",
      "Nearest to 西湖:  拱券,  灯火阑珊,  自然保护区,  数,  整个,  大修,  留,  洞洞, \n",
      "Nearest to 最:  的,  武当,  朝拜,  艺人,  全部,  非常,  没赶上,  保护, \n",
      "Nearest to 有:  还有,  的,  是,  鎏金,  珍稀动物,  在,  粑粑,  心理, \n",
      "Nearest to 但:  的,  捐,  设立,  死人,  老街,  和,  喜欢,  方特, \n",
      "Nearest to 这里:  有机,  长沙,  不收,  钙,  巅,  银子,  辉,  擦擦, \n",
      "Nearest to 两:  乡试,  一,  附件,  中秋,  几,  自古以来,  瑰宝,  套, \n",
      "Nearest to 不错:  很好,  古国,  书楼,  鬼,  景色,  擦擦,  挺好,  风景, \n",
      "Nearest to 特别:  超级,  很,  比赛,  非常,  而且,  山谷,  口子,  帅哥, \n",
      "Nearest to 看看:  耳熟能详,  石景山,  扬州,  擦擦,  分外,  爱情,  杆子,  啊, \n",
      "Nearest to 了:  的,  UNK,  就,  摇橹船,  外线,  是,  消息,  五指山市, \n",
      "Nearest to 能:  可以,  平台,  算得,  参天大树,  吉普车,  住宅区,  滑板,  我们, \n",
      "Nearest to 因为:  总投资,  擦擦,  作为,  也,  快照,  中药材,  黑马河乡,  120, \n",
      "Nearest to 不是:  宗教,  是,  餐,  的,  后坐,  开玩笑,  庞大,  很长, \n",
      "Average loss at step 22000 :  7.551067021727562\n",
      "Average loss at step 24000 :  7.262484151482582\n",
      "Average loss at step 26000 :  6.788807342767716\n",
      "Average loss at step 28000 :  6.466453863143921\n",
      "Average loss at step 30000 :  6.2299574464559555\n",
      "Nearest to 等:  和,  z,  很多,  人造,  琼台,  微电影,  大兴土木,  开船, \n",
      "Nearest to 去:  来,  擦擦,  引桥,  到,  二级,  转车,  进去,  美图, \n",
      "Nearest to 这个:  其他,  黄线,  外人,  观看,  雪花,  省,  功略,  十分, \n",
      "Nearest to 西湖:  拱券,  灯火阑珊,  自然保护区,  整个,  数,  洞洞,  里能,  大修, \n",
      "Nearest to 最:  非常,  朝拜,  武当,  的,  保护,  感受,  很,  纪晓岚, \n",
      "Nearest to 有:  还有,  珍稀动物,  揣,  的,  在,  心理,  黄琉璃,  擦擦, \n",
      "Nearest to 但:  但是,  龙进溪,  啊,  捐,  方特,  不过,  凤冠,  死人, \n",
      "Nearest to 这里:  有机,  长沙,  钙,  不收,  银子,  我们,  美得,  巅, \n",
      "Nearest to 两:  一,  几,  乡试,  附件,  中秋,  2,  套,  反, \n",
      "Nearest to 不错:  很好,  挺好,  古国,  引起,  开心,  书楼,  景色,  鬼, \n",
      "Nearest to 特别:  很,  超级,  非常,  而且,  口子,  比赛,  炭厂村,  山谷, \n",
      "Nearest to 看看:  石景山,  耳熟能详,  扬州,  擦擦,  杆子,  分外,  啊,  看着, \n",
      "Nearest to 了:  的,  消息,  就,  三河,  擦擦,  外线,  摇橹船,  过, \n",
      "Nearest to 能:  可以,  算得,  平台,  参天大树,  没,  滑板,  我们,  自作主张, \n",
      "Nearest to 因为:  也,  擦擦,  总投资,  信函,  UNK,  周末,  作为,  Aa, \n",
      "Nearest to 不是:  是,  宗教,  餐,  开玩笑,  后坐,  车堵,  很长,  比, \n",
      "Average loss at step 32000 :  6.175477685332298\n",
      "Average loss at step 34000 :  5.813753595292568\n",
      "Average loss at step 36000 :  5.723748626708985\n",
      "Average loss at step 38000 :  5.682567628264427\n",
      "Average loss at step 40000 :  5.504575249433517\n",
      "Nearest to 等:  和,  z,  很多,  人造,  大兴土木,  微电影,  吹发,  开船, \n",
      "Nearest to 去:  来,  擦擦,  引桥,  到,  进去,  美图,  玉关,  转车, \n",
      "Nearest to 这个:  其他,  宝冠,  黄线,  外人,  雪花,  观看,  省,  西泠桥, \n",
      "Nearest to 西湖:  拱券,  灯火阑珊,  整个,  五女峰,  热爱,  数,  里能,  一蹭, \n",
      "Nearest to 最:  非常,  很,  保护,  朝拜,  更,  纪晓岚,  武当,  感受, \n",
      "Nearest to 有:  还有,  珍稀动物,  心理,  的,  葱茏,  揣,  擦擦,  大小便, \n",
      "Nearest to 但:  但是,  不过,  龙进溪,  啊,  凤冠,  死人,  抓狂,  因为, \n",
      "Nearest to 这里:  有机,  长沙,  那里,  扭扭捏捏,  钙,  周末,  这边,  我们, \n",
      "Nearest to 两:  一,  几,  三,  2,  乡试,  附件,  好几,  百态, \n",
      "Nearest to 不错:  很好,  挺好,  古国,  开心,  引起,  竹竹,  书楼,  鬼, \n",
      "Nearest to 特别:  超级,  很,  非常,  而且,  口子,  挺,  炭厂村,  防风, \n",
      "Nearest to 看看:  耳熟能详,  石景山,  扬州,  杆子,  擦擦,  看着,  分外,  竹竹, \n",
      "Nearest to 了:  的,  竹竹,  三河,  就,  啊,  消息,  久旱,  真累, \n",
      "Nearest to 能:  可以,  没,  参天大树,  算得,  平台,  滑板,  无辜,  才能, \n",
      "Nearest to 因为:  也,  擦擦,  但,  周末,  Aa,  不过,  最起码,  经幡, \n",
      "Nearest to 不是:  宗教,  是,  开玩笑,  车堵,  后坐,  比,  餐,  但是, \n",
      "Average loss at step 42000 :  5.405763406395912\n",
      "Average loss at step 44000 :  5.336656320810318\n",
      "Average loss at step 46000 :  5.242426370501518\n",
      "Average loss at step 48000 :  5.085996184945106\n",
      "Average loss at step 50000 :  4.828027547955513\n",
      "Nearest to 等:  和,  z,  很多,  UNK,  人造,  开船,  银鱼,  大兴土木, \n",
      "Nearest to 去:  来,  擦擦,  到,  进去,  地方挺大,  出行,  引桥,  玉关, \n",
      "Nearest to 这个:  其他,  宝冠,  黄线,  这里,  观看,  一个,  连,  美轮美奂, \n",
      "Nearest to 西湖:  拱券,  灯火阑珊,  热爱,  整个,  五女峰,  里能,  700,  数, \n",
      "Nearest to 最:  非常,  保护,  很,  更,  朝拜,  诗词,  十分,  纪晓岚, \n",
      "Nearest to 有:  还有,  心理,  珍稀动物,  的,  葱茏,  揣,  黄琉璃,  擦擦, \n",
      "Nearest to 但:  但是,  不过,  啊,  龙进溪,  所以,  虽然,  因为,  Aa, \n",
      "Nearest to 这里:  有机,  那里,  长沙,  这边,  扭扭捏捏,  钙,  此,  擦擦, \n",
      "Nearest to 两:  几,  一,  三,  2,  好几,  乡试,  附件,  3, \n",
      "Nearest to 不错:  很好,  挺好,  感觉,  开心,  引起,  书楼,  竹竹,  擦擦, \n",
      "Nearest to 特别:  非常,  超级,  很,  而且,  口子,  挺,  比较,  防风, \n",
      "Nearest to 看看:  石景山,  耳熟能详,  看,  扬州,  擦擦,  看着,  杆子,  紧闭, \n",
      "Nearest to 了:  的,  竹竹,  轩辕台,  啊,  真累,  寄给,  擦擦,  消息, \n",
      "Nearest to 能:  可以,  没,  参天大树,  才能,  能够,  滑板,  会,  平台, \n",
      "Nearest to 因为:  也,  但,  擦擦,  不过,  最起码,  如果,  Aa,  周末, \n",
      "Nearest to 不是:  是,  宗教,  很长,  山和云,  后坐,  开玩笑,  梨木,  大佛像, \n",
      "Average loss at step 52000 :  4.758704661026597\n",
      "Average loss at step 54000 :  4.694996470235288\n",
      "Average loss at step 56000 :  4.707180620193482\n",
      "Average loss at step 58000 :  4.693072471022606\n",
      "Average loss at step 60000 :  4.695204861164093\n",
      "Nearest to 等:  和,  z,  很多,  人造,  吹发,  银鱼,  开船,  大兴土木, \n",
      "Nearest to 去:  来,  擦擦,  进去,  过去,  到,  竹竹,  地方挺大,  游玩, \n",
      "Nearest to 这个:  其他,  宝冠,  观看,  一个,  这里,  美轮美奂,  黄线,  连, \n",
      "Nearest to 西湖:  拱券,  灯火阑珊,  热爱,  整个,  五女峰,  杭州,  700,  峭, \n",
      "Nearest to 最:  非常,  保护,  最为,  很,  更,  十分,  梁柱,  朝拜, \n",
      "Nearest to 有:  还有,  珍稀动物,  心理,  揣,  屯,  聚餐,  葱茏,  黄琉璃, \n",
      "Nearest to 但:  但是,  不过,  虽然,  龙进溪,  啊,  所以,  凤冠,  因为, \n",
      "Nearest to 这里:  那里,  有机,  这边,  扭扭捏捏,  钙,  里面,  长沙,  此, \n",
      "Nearest to 两:  几,  一,  三,  2,  好几,  附件,  3,  百态, \n",
      "Nearest to 不错:  很好,  挺好,  竹竹,  开心,  引起,  感觉,  古国,  书楼, \n",
      "Nearest to 特别:  非常,  超级,  很,  而且,  比较,  挺,  防风,  口子, \n",
      "Nearest to 看看:  看,  石景山,  擦擦,  耳熟能详,  扬州,  看着,  竹竹,  走走, \n",
      "Nearest to 了:  竹竹,  轩辕台,  的,  消息,  久旱,  就,  着,  擦擦, \n",
      "Nearest to 能:  可以,  没,  才能,  能够,  会,  参天大树,  平台,  滑板, \n",
      "Nearest to 因为:  不过,  也,  但,  擦擦,  如果,  最起码,  Aa,  周末, \n",
      "Nearest to 不是:  很长,  但是,  宗教,  没有,  是,  山和云,  后坐,  开玩笑, \n",
      "Average loss at step 62000 :  4.678471186161041\n",
      "Average loss at step 64000 :  4.657299560844899\n",
      "Average loss at step 66000 :  4.658523216366768\n",
      "Average loss at step 68000 :  4.6366935933828355\n",
      "Average loss at step 70000 :  4.640592899501324\n",
      "Nearest to 等:  和,  z,  很多,  人造,  吹发,  银鱼,  大兴土木,  微电影, \n",
      "Nearest to 去:  来,  擦擦,  进去,  过去,  过来,  地方挺大,  美图,  转车, \n",
      "Nearest to 这个:  其他,  这里,  一个,  观看,  连,  宝冠,  美轮美奂,  西泠桥, \n",
      "Nearest to 西湖:  拱券,  灯火阑珊,  热爱,  整个,  五女峰,  一蹭,  杭州,  700, \n",
      "Nearest to 最:  非常,  保护,  很,  最为,  更,  十分,  朝拜,  出席, \n",
      "Nearest to 有:  还有,  心理,  珍稀动物,  黄琉璃,  揣,  徐州,  的,  屯, \n",
      "Nearest to 但:  但是,  不过,  所以,  虽然,  龙进溪,  凤冠,  啊,  而且, \n",
      "Nearest to 这里:  那里,  这边,  里面,  有机,  长沙,  此,  扭扭捏捏,  钙, \n",
      "Nearest to 两:  几,  三,  一,  2,  好几,  3,  第二,  罗浮山, \n",
      "Nearest to 不错:  很好,  挺好,  竹竹,  感觉,  引起,  开心,  很漂亮,  古国, \n",
      "Nearest to 特别:  非常,  超级,  很,  而且,  比较,  挺,  那么,  防风, \n",
      "Nearest to 看看:  看,  石景山,  耳熟能详,  扬州,  看着,  擦擦,  走走,  竹竹, \n",
      "Nearest to 了:  竹竹,  轩辕台,  的,  消息,  啊,  卸下,  不了,  摇橹船, \n",
      "Nearest to 能:  可以,  才能,  没,  会,  能够,  参天大树,  清王朝,  滑板, \n",
      "Nearest to 因为:  不过,  如果,  但,  也,  擦擦,  Aa,  其实,  当时, \n",
      "Nearest to 不是:  很长,  但是,  没有,  梨木,  宗教,  只是,  山和云,  开玩笑, \n",
      "Average loss at step 72000 :  4.632153640627861\n",
      "Average loss at step 74000 :  4.610035398721695\n",
      "Average loss at step 76000 :  4.601241475224495\n",
      "Average loss at step 78000 :  4.611391437292099\n",
      "Average loss at step 80000 :  4.603660498321056\n",
      "Nearest to 等:  z,  和,  人造,  吹发,  银鱼,  大兴土木,  很多,  微电影, \n",
      "Nearest to 去:  来,  擦擦,  进去,  过来,  过去,  住,  二级,  地方挺大, \n",
      "Nearest to 这个:  其他,  这里,  一个,  观看,  宝冠,  连,  西泠桥,  美轮美奂, \n",
      "Nearest to 西湖:  拱券,  灯火阑珊,  杭州,  五女峰,  热爱,  峭,  整个,  一蹭, \n",
      "Nearest to 最:  非常,  最为,  保护,  很,  更,  十分,  朝拜,  纪晓岚, \n",
      "Nearest to 有:  还有,  大小便,  珍稀动物,  心理,  的,  黄琉璃,  揣,  屯, \n",
      "Nearest to 但:  但是,  不过,  所以,  虽然,  而且,  龙进溪,  啊,  凤冠, \n",
      "Nearest to 这里:  这边,  那里,  有机,  此,  长沙,  里面,  扭扭捏捏,  这个, \n",
      "Nearest to 两:  几,  三,  一,  2,  好几,  3,  第二,  罗浮山, \n",
      "Nearest to 不错:  很好,  挺好,  感觉,  值得一看,  开心,  竹竹,  轩辕台,  书楼, \n",
      "Nearest to 特别:  非常,  超级,  很,  而且,  比较,  挺,  那么,  防风, \n",
      "Nearest to 看看:  看,  看着,  石景山,  走走,  耳熟能详,  扬州,  擦擦,  杆子, \n",
      "Nearest to 了:  轩辕台,  竹竹,  消息,  啊,  不了,  的,  啦,  擦擦, \n",
      "Nearest to 能:  可以,  能够,  会,  才能,  没,  参天大树,  什么鸟,  锅, \n",
      "Nearest to 因为:  不过,  当时,  但,  其实,  也,  擦擦,  如果,  由于, \n",
      "Nearest to 不是:  很长,  但是,  抽走,  没有,  梨木,  只是,  宗教,  开玩笑, \n",
      "Average loss at step 82000 :  4.5865989966392515\n",
      "Average loss at step 84000 :  4.577741507649422\n",
      "Average loss at step 86000 :  4.588486400842666\n",
      "Average loss at step 88000 :  4.567631370544434\n",
      "Average loss at step 90000 :  4.555695701122284\n",
      "Nearest to 等:  z,  和,  吹发,  人造,  银鱼,  大兴土木,  开船,  就地, \n",
      "Nearest to 去:  来,  擦擦,  进去,  过来,  过去,  二级,  想去,  出行, \n",
      "Nearest to 这个:  其他,  这里,  一个,  宝冠,  连,  观看,  这,  西泠桥, \n",
      "Nearest to 西湖:  拱券,  灯火阑珊,  杭州,  五女峰,  热爱,  确立,  整个,  峭, \n",
      "Nearest to 最:  最为,  非常,  保护,  更,  很,  十分,  出席,  纪晓岚, \n",
      "Nearest to 有:  还有,  心理,  珍稀动物,  揣,  大小便,  擦擦,  葱茏,  批发市场, \n",
      "Nearest to 但:  但是,  不过,  所以,  虽然,  而且,  龙进溪,  啊,  当然, \n",
      "Nearest to 这里:  那里,  这边,  此,  有机,  长沙,  里面,  现在,  这, \n",
      "Nearest to 两:  几,  三,  一,  2,  好几,  3,  第二,  罗浮山, \n",
      "Nearest to 不错:  很好,  挺好,  值得一看,  开心,  竹竹,  很棒,  感觉,  轩辕台, \n",
      "Nearest to 特别:  非常,  超级,  很,  比较,  而且,  挺,  那么,  尤其, \n",
      "Nearest to 看看:  看,  看着,  走走,  耳熟能详,  逛逛,  擦擦,  石景山,  欣赏, \n",
      "Nearest to 了:  啊,  竹竹,  轩辕台,  久旱,  不了,  就,  呢,  卸下, \n",
      "Nearest to 能:  可以,  能够,  才能,  会,  没,  参天大树,  文化村,  不能, \n",
      "Nearest to 因为:  不过,  当时,  由于,  其实,  如果,  但,  擦擦,  也, \n",
      "Nearest to 不是:  很长,  没有,  抽走,  但是,  梨木,  开玩笑,  宗教,  山和云, \n",
      "Average loss at step 92000 :  4.55447738134861\n",
      "Average loss at step 94000 :  4.548984165787697\n",
      "Average loss at step 96000 :  4.526846668243408\n",
      "Average loss at step 98000 :  4.502312752246857\n",
      "Average loss at step 100000 :  4.460730201154948\n",
      "Nearest to 等:  z,  和,  人造,  吹发,  银鱼,  开船,  大兴土木,  违反规定, \n",
      "Nearest to 去:  来,  擦擦,  过来,  过去,  进去,  地方挺大,  想去,  竹竹, \n",
      "Nearest to 这个:  其他,  这里,  一个,  连,  宝冠,  这,  这边,  每个, \n",
      "Nearest to 西湖:  灯火阑珊,  拱券,  杭州,  热爱,  确立,  五女峰,  太湖,  湖边, \n",
      "Nearest to 最:  最为,  非常,  保护,  很,  更,  十分,  出席,  纪晓岚, \n",
      "Nearest to 有:  还有,  心理,  批发市场,  珍稀动物,  揣,  Hello,  葱茏,  屯, \n",
      "Nearest to 但:  但是,  不过,  虽然,  而且,  所以,  龙进溪,  当然,  啊, \n",
      "Nearest to 这里:  那里,  这边,  有机,  此,  里面,  现在,  长沙,  杭州, \n",
      "Nearest to 两:  几,  三,  一,  2,  好几,  3,  第二,  罗浮山, \n",
      "Nearest to 不错:  很好,  挺好,  很棒,  开心,  值得一看,  感觉,  竹竹,  轩辕台, \n",
      "Nearest to 特别:  超级,  非常,  很,  比较,  而且,  挺,  那么,  尤其, \n",
      "Nearest to 看看:  看,  看着,  走走,  逛逛,  欣赏,  耳熟能详,  擦擦,  石景山, \n",
      "Nearest to 了:  竹竹,  轩辕台,  啊,  啦,  消息,  寄给,  不了,  的雪乡, \n",
      "Nearest to 能:  可以,  能够,  才能,  会,  没,  不能,  什么鸟,  文化村, \n",
      "Nearest to 因为:  不过,  由于,  当时,  其实,  虽然,  如果,  逛完,  最起码, \n",
      "Nearest to 不是:  没有,  很长,  抽走,  山和云,  但是,  后坐,  比做,  梨木, \n"
     ]
    }
   ],
   "source": [
    "num_steps = 100001\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    init.run()\n",
    "    print('Initialized')\n",
    "    \n",
    "    average_loss = 0\n",
    "    for step in range(num_steps):\n",
    "        batch_inputs, batch_labels = generate_batch(\n",
    "            batch_size, num_skips, skip_window)\n",
    "        feed_dict = {train_inputs : batch_inputs, train_labels : batch_labels}\n",
    "        _, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)\n",
    "        average_loss += loss_val\n",
    "        if step % 2000 == 0:\n",
    "            if step > 0:\n",
    "                average_loss /= 2000\n",
    "            print('Average loss at step', step, ': ', average_loss)\n",
    "            average_loss = 0\n",
    "        if step % 10000 == 0:\n",
    "            sim = similarity.eval()\n",
    "            for i in range(valid_size):\n",
    "                valid_word = reverse_dictionary[valid_examples[i]]\n",
    "                top_k = 8\n",
    "                nearest = (-sim[i, :]).argsort()[1:top_k + 1]\n",
    "                log_str = 'Nearest to % s: ' % valid_word\n",
    "                for k in range(top_k):\n",
    "                    closed_word = reverse_dictionary[nearest[k]]\n",
    "                    log_str = \"%s %s, \" % (log_str, closed_word)\n",
    "                print(log_str)\n",
    "    final_embeddings = normalized_embeddings.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
